from bs4 import BeautifulSoup
import requests
import os
import re
import time
import csv
### ###############################################
### 如果使用excel打开生成的csv文件，可能会存在乱码       
### 的状况，建议使用wps；如果一定要使用excel来打开，
### 需要通过以下几步：   打开excel文档 -> 数据 
### -> 自文本 -> 选择csv文件 -> 下一步 -> 逗号分隔 
### -> 完成
### ########################################### ###


class Line(object):
    def __init__(self, username, province, product, logo, catelog, year, car, reason, sub_reason, create_time):
        self.username = username
        self.province = province
        self.product = product
        self.logo = logo
        self.catelog = catelog
        self.year = year
        self.car = car
        self.reason = reason
        self.sub_reason = sub_reason
        self.create_time = create_time

    def toString(self):
        return '%s,%s,%s,%s,%s,%s,%s,%s,%s,%s' % (self.username, self.province,self.product, self.logo, self.catelog, self.year, self.car, self.reason, self.sub_reason, self.create_time)

URL = 'http://www.dpac.gov.cn/cpqxcj/VehicleComplaintPublicity.aspx'
HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.101 Safari/537.36',
    'Content-Type': 'application/x-www-form-urlencoded',
    'ASP.NET_SessionId': 'ASP.NET_SessionId=zp2mf2qjmhdcls55oq1ho4z3; __jsluid=a38e14d658367ffaec7e441f5768bce9; cck_lasttime=1504059744203; cck_count=0'
}
CURRENT_DIR = os.path.realpath(__file__)
FILES_STORE = os.path.join(CURRENT_DIR, '..', 'output')
csv_header = None
has_write_header = False

def main():
    start = time.time()
    pages = fetch_pages()
    print('共%s页,共%s条' % (pages[0], pages[1]))
    for no in range(0, int(pages[0])):
        lines = downloadPage(no)
        writeLines(lines)
    print('下载完成，耗时%.2f秒' % (time.time() - start))


def fetch_pages():
    print('正在爬取页码信息...')
    values = {
        '__EVENTARGUMENT': 1
    }
    req = requests.post(URL, data = values, headers = HEADERS)
    soup = BeautifulSoup(req.text, 'html.parser')
    page_info = soup.select('.pn')[0]
    return re.findall(r'\d+', page_info.text.strip())
    
def downloadPage(pageNo = 1):
    print('正在爬取第%s页'%(pageNo,))
    postdata = {
        '__VIEWSTATE': '/wEPDwUKMTE1NDY4NDUwNQ9kFgICAw9kFgQCAQ8WAh4LXyFJdGVtQ291bnQCDxYeZg9kFgICBA8VCQnlsbHkuJznnIE15bm/5rG96I+y5Lqa54m55YWL6I6x5pav5YuS5rG96L2m5pyJ6ZmQ5YWs5Y+4KOWbveS6pykQ5ZCJ5pmu77yISkVFUO+8iQ/oh6rnlLHlhYnns7vliJcEMjAxNg0yLjRM6aKG5YWI54mICeWPkeWKqOacug/msb3msrnlj5HliqjmnLoSMjAxNy04LTI5IDE1OjIyOjA5ZAIBD2QWAgIEDxUJCeWMl+S6rOW4girlpKfkvJfmsb3ovabvvIjkuK3lm73vvInplIDllK7mnInpmZDlhazlj7gWVm9sa3N3YWdlbu+8iOWkp+S8l++8iQbpgJTplJAEMjAxNhjpgJTplJDmsb3msrnniYjoiJLpgILniYgJ5Y+R5Yqo5py6DOi/m+awlOezu+e7nxEyMDE3LTgtMjkgOToyODo0NmQCAg9kFgICBA8VCQnmtZnmsZ/nnIEn5LiA5rG9LeWkp+S8l+axvei9puaciemZkOWFrOWPuCjlm73kuqcpDOS4gOaxveWlpei/qgJBMwQyMDE0LjIwMTQvMjAxNeasvkEzIFNQT1JUQkFDSyAzNVRGU0noh6rliqjoiJLpgILlnosJ5Lyg5Yqo57O7B0RTRy9EQ1QSMjAxNy04LTI4IDIzOjQ0OjAxZAIDD2QWAgIEDxUJCeW5v+S4nOecgTXlub/msb3oj7LkuprnibnlhYvojrHmlq/li5Lmsb3ovabmnInpmZDlhazlj7go5Zu95LqnKQ/lub/msb3oj7LkuprnibkM6I+y57+U57O75YiXBDIwMTZI6I+y57+UMjAxNeasvjEuNFQxNTBocCBERENU77yI5pm65Lqr54mI44CB5bCK5Lqr54mI44CB6LeD5Lqr54mI77yJ5Zu95LqUCeS8oOWKqOezuwdEU0cvRENUEjIwMTctOC0yOCAyMzo0MTowOGQCBA9kFgICBA8VCQnlm5vlt53nnIEk5LiK5rG96YCa55So5rG96L2m6ZSA5ZSu5pyJ6ZmQ5YWs5Y+4GkNhZGlsbGFj77yI5Yev6L+q5ouJ5YWL77yJA1NSWAQyMDEzETMuMEwg6LGq5Y2O54mIIEFUDOeUteawlOiuvuWkhwbku6rooagSMjAxNy04LTI4IDIzOjIwOjI1ZAIFD2QWAgIEDxUJCeWxseS4nOecgSbljJfkuqzlpZTpqbDmsb3ovabmnInpmZDlhazlj7go5Zu95LqnKQzljJfkuqzlpZTpqbAEQ+e6pwQyMDE1FkMgMjYwIEwg77yIQko3MjA0RkzvvIkJ5Yi25Yqo57O7Eua2suWOi+ihjOi9puWItuWKqBIyMDE3LTgtMjggMjM6MTM6MTlkAgYPZBYCAgQPFQkJ5rWZ5rGf55yBHuS4iuaxvemAmueUqOaxvei9puaciemZkOWFrOWPuBjkuIrmsb3pgJrnlKjlh6/ov6rmi4nlhYsDWFRTBDIwMTQPMjhUIOiIkumAguWei0FUDOeUteawlOiuvuWkhwzovabouqvmqKHlnZcSMjAxNy04LTI4IDIyOjUwOjA4ZAIHD2QWAgIEDxUJCeays+WNl+ecgSblub/msb3mnKznlLDmsb3ovabmnInpmZDlhazlj7go5Zu95LqnKQzlub/msb3mnKznlLAG6ZuF6ZiBBDIwMTQbMjAxNC8yMDE15qy+IDIuMEVY6LGq5Y2O54mIBui9pui6qwbovabmnrYSMjAxNy04LTI4IDIyOjE2OjUyZAIID2QWAgIEDxUJCeW5v+S4nOecgR7kuIrmsb3pgJrnlKjmsb3ovabmnInpmZDlhazlj7gY5LiK5rG96YCa55So5Yev6L+q5ouJ5YWLA1hUUwQyMDEzDzI4VCDoiJLpgILlnotBVAznlLXmsJTorr7lpIcG5a+86IiqEjIwMTctOC0yOCAyMDozNTowNWQCCQ9kFgICBA8VCQnlsbHkuJznnIEe5LiK5rG95aSn5LyX5rG96L2m5pyJ6ZmQ5YWs5Y+4DOS4iuaxveWkp+S8lwbpgJTop4IEMjAxNi3lhajmlrDpgJTop4IxLjhUU0nmiYvoh6rkuIDkvZPoiJLpgILniYjliY3pqbEM6ZmE5Yqg6K6+5aSHBuacuuaisBIyMDE3LTgtMjggMjA6MDk6NTJkAgoPZBYCAgQPFQkJ5rGf6IuP55yBHuS4iuaxvemAmueUqOaxvei9puaciemZkOWFrOWPuBjkuIrmsb3pgJrnlKjlh6/ov6rmi4nlhYsDWFRTBDIwMTQPMjhUIOiIkumAguWei0FUDOeUteawlOiuvuWkhwbku6rooagSMjAxNy04LTI4IDE5OjI3OjE1ZAILD2QWAgIEDxUJCeS4iua1t+W4gifkuIDmsb0t5aSn5LyX5rG96L2m5pyJ6ZmQ5YWs5Y+4KOWbveS6pykN5LiA5rG9LeWkp+S8lw3pgJ/ohb5TQUdJVEFSBDIwMTEf6YCf6IW+MS40VFNJIERTR+iHquWKqOeyvuiLseWeiwnkvKDliqjns7sHRFNHL0RDVBIyMDE3LTgtMjggMTk6MTY6NDRkAgwPZBYCAgQPFQkJ5bm/5Lic55yBHumVv+Wuieemj+eJueaxvei9puaciemZkOWFrOWPuAzplb/lronnpo/nibkM5paw56aP5YWL5pavBDIwMTUk5paw56aP5YWL5pavIOS4pOWOoiAxLjYgQVQg6aOO5bCa5Z6LCeS8oOWKqOezuw/oh6rliqjlj5jpgJ/lmagSMjAxNy04LTI4IDE4OjQ4OjU5ZAIND2QWAgIEDxUJCemZleilv+ecgSrlpKfkvJfmsb3ovabvvIjkuK3lm73vvInplIDllK7mnInpmZDlhazlj7gWVm9sa3N3YWdlbu+8iOWkp+S8l++8iQbpgJTplJAEMjAxNxjpgJTplJDmsb3msrnniYjoiJLpgILniYgJ5Y+R5Yqo5py6DOi/m+awlOezu+e7nxIyMDE3LTgtMjggMTg6Mzk6NDVkAg4PZBYCAgQPFQkJ5a6J5b6955yBHuS4iuaxvemAmueUqOaxvei9puaciemZkOWFrOWPuBjkuIrmsb3pgJrnlKjlh6/ov6rmi4nlhYsDWFRTBDIwMTMPMjhUIOiIkumAguWei0FUDOeUteawlOiuvuWkhwzovabouqvmqKHlnZcSMjAxNy04LTI4IDE4OjI0OjE2ZAIDDw8WAh4LUmVjb3JkY291bnQC1QlkZGQ6NU3+uapqHLYxnmEjUhA2PZ/lAQ==',
        '__EVENTTARGET': 'pagelist',
        '__EVENTARGUMENT': pageNo
    }
    
    req = requests.post(URL, data = postdata, headers = HEADERS)
    soup = BeautifulSoup(req.text, 'html.parser')
    datas = soup.select('.dataList tr')
    print('共有%s条数据'% (len(datas[1:]),))
    global csv_header
    if csv_header == None:
        header = list(map(lambda col: col.text.strip() + ',', [col for col in datas[0].select('th')]))
        csv_header = ''.join(header[:-1]) + '\n'
    results = []
    for line in datas[1:]:
        lo = parseLineToObj(line)
        results.append(lo)
    return results
    
def parseLineToObj(line):
    columns = line.select('td')
    lo = Line(
        *list(map(lambda col: col.text.strip(), [col for col in columns]))
    )
    return lo
def writeLines(lines = []):
    if not os.path.isdir(FILES_STORE):
        os.makedirs(FILES_STORE)

    with open(os.path.join(FILES_STORE, 'car.csv'), 'a+', encoding='UTF-8') as f:
        global has_write_header
        if not has_write_header:
            f.write(csv_header)
            has_write_header = True
        for line in lines:
            f.write(line.toString() + '\n');
main()
